Naver Open API
Products -> 서비스 API -> 검색 (https://developers.naver.com/products/service-api/search/search.md)
We are not using .csv file toady.
API (Application Programming Interface): Mechanism or system enabling two software components to communicate using a set of definitions and protocols. (https://aws.amazon.com/what-is/api/ https://brunch.co.kr/@jhw28/31)
Let’s look at the address of CAU Notice page.
We are using NAVER Open API today. To use an API, you have to read the API reference. It’s sort of a dictionary.
https://developers.naver.com/docs/serviceapi/search/news/news.md#%EB%89%B4%EC%8A%A4
XML (Extensible Markup Language): Language used to define and store data in a sharable manner. ( https://aws.amazon.com/what-is/xml/?nc1=h_ls, https://www.tcpschool.com/xml/xml_basic_structure)
library(dplyr)##
## Attaching package: 'dplyr'## The following objects are masked from 'package:stats':
##
## filter, lag## The following objects are masked from 'package:base':
##
## intersect, setdiff, setequal, unionlibrary(httr)
library(XML)Select your word to search.
keyword <- "검색하고싶으은문자열을넣으세요"Using API, we need to communicate with NAVER the But, URL doesnt not understand 한글. Just like the NAVER API reference document address above, it should be in Percent Encoding. “%” sign does not mean percentage. https://www.youtube.com/watch?v=3DNaj8R4HJg&t=37s
(https://it-eldorado.tistory.com/143, https://bunny.net/academy/http/what-is-url-uniform-resource-identifier-and-percent-encoding/)
myQuery <- keyword %>%
enc2utf8() %>%
URLencode()
print(myQuery)## [1] "%EC%97%AC%ED%96%89"We are making an order to API server. 1) language: percent encoding. 2) vocabulary: read the reference.
searchUrl <- "https://openapi.naver.com/v1/search/news.xml"
chunk <- 50
startNumber <-1
sort<- "sim"
url <- glue::glue("{searchUrl}?query={myQuery}&display={chunk}&start={startNumber}&sort={sort}")
url## https://openapi.naver.com/v1/search/news.xml?query=%EC%97%AC%ED%96%89&display=50&start=1&sort=simGET(url, add_header(information to request) : retrieves
the page with the url
doc_raw <- url %>%
GET(
add_headers(
"X-Naver-Client-Id" = client_id,
"X-Naver-Client-Secret" = client_secret
)
) %>%
toString() %>%
XML::xmlParse()Let’s see how doc_raw looks like.
It’s well organized but hard to read. Let’s make it into a dataframe. One row for one article.
doc_df<- doc_raw %>%
getNodeSet("//item") %>%
xmlToDataFrame()
head(doc_df)As NAVER News search result shows my keyword in bold, the reuslt also
has <b></b> html tags. We really do not neet
it. Let’s remove it.
For that, we are using regular expression, which is also a way of present letters/numbers/punctuation and stuff.
https://www.youtube.com/shorts/kGFUu5WAzd8
library(stringr)
doc_df%>%mutate(title_text = str_remove_all(
title, "&\\w+;|<[[:punct:]]*b>"))Similarly, let’s define a function to refine the texts.
get_list <- function(doc) {
refined<- doc %>%
getNodeSet("//item") %>%
xmlToDataFrame() %>%
rename("publish_date" = pubDate) %>%
mutate(publish_date = as.POSIXct(publish_date,
format = "%a, %d %b %Y %H:%M:%S %z")) %>%
mutate(title_text = str_replace_all(
title, "&\\w+;|<[[:punct:]]*b>", " ")) %>%
mutate(title_text = str_replace_all(
title_text, "([[:punct:]])", " ")) %>%
mutate(title_text = str_replace_all(
title_text, "\\s{2,}", " ")) %>%
mutate(title_text = str_remove_all(
title_text, "\\A\\s+")) %>%
mutate(description_text = str_remove_all(
description,
"&\\w+;|<[[:punct:]]*b>|[“”]"))
return(refined)
}get_list(doc_raw)Let’s say we want to get 100 result for “첫눈” and 500 results for “채용”. What do we do when repeating same process? Use a function :)
NNSearch <- function(keyword = NULL, chunk = 100, start = 1,
sort = c("date", "sim"), do_done = FALSE, max = 1000L,
client_id = client_id, client_secret = client_secret,
verbose = TRUE) {
if (is.null(keyword)) {
stop("Error: Input search keyword")
}
if (chunk < 1 & chunk > 100) {
stop("Error: Chunk size should be 1 < chunk <=100")
}
if (start < 1 & start > 1000) {
stop("Error: Start should be 1< start <=1000.")
}
sort <- match.arg(sort)
searchUrl <- "https://openapi.naver.com/v1/search/news.xml"
query <- keyword %>%
enc2utf8() %>%
URLencode()
url <- glue::glue("{searchUrl}?query={query}&display={chunk}&start={start}&sort={sort}")
doc <- url %>%
GET(
add_headers(
"X-Naver-Client-Id" = client_id,
"X-Naver-Client-Secret" = client_secret
)
) %>%
toString() %>%
XML::xmlParse()
totalNaverResult <- doc %>%
XML::getNodeSet("//total") %>%
XML::xmlValue() %>%
as.integer()
if (!do_done){
max <- chunk
}
if (verbose) {
glue::glue("* [{keyword}] 뉴스 검색 결과: 총 {totalNaverResult}건.\n\n") %>%
cat()
glue::glue(" - ({chunk}/{min(totalNaverResult, max)})건 완료.\n\n") %>%
cat()
}
search_list <- doc %>%
get_list()
nRecord <- NROW(search_list)
if (!do_done | nRecord >= totalNaverResult | nRecord >= max) {
return(search_list)
} else {
totalMyResult <- min(totalNaverResult, max)
pageCount <- totalMyResult %/% chunk
if (totalMyResult %/% chunk == 0) {
pageCount <- pageCount - 1
}
idx <- (seq(pageCount-1)+1)
add_list <- idx[idx <= 1000] %>%
purrr::map_df({
function(x) {
if (verbose) {
glue::glue(" - ({chunk * x}/{totalMyResult})건 .\n\n") %>%
cat()
}
glue::glue(
"{searchUrl}?query={query}&display={chunk}&start={x}&sort={sort}"
) %>%
httr::GET(
httr::add_headers(
"X-Naver-Client-Id" = client_id,
"X-Naver-Client-Secret" = client_secret
)
) %>%
toString() %>%
XML::xmlParse() %>%
get_list()
}
})
search_list %>%
bind_rows(
add_list
) %>%
return()
}
}firstsnow<- NNSearch("첫눈", client_id = client_id, client_secret = client_secret,
sort="sim", max=100, do_done=T)## * [첫눈] 뉴스 검색 결과: 총 120463건.
## - (100/100)건 완료.hiring_sim<- NNSearch("채용", client_id = client_id, client_secret = client_secret,
sort="sim", max=1000, do_done=T)## * [채용] 뉴스 검색 결과: 총 2730527건.
## - (100/1000)건 완료.
## - (200/1000)건 .
## - (300/1000)건 .
## - (400/1000)건 .
## - (500/1000)건 .
## - (600/1000)건 .
## - (700/1000)건 .
## - (800/1000)건 .
## - (900/1000)건 .
## - (1000/1000)건 .hiring_date<- NNSearch("채용", client_id = client_id, client_secret = client_secret,
sort="date", max=1000, do_done=T)## * [채용] 뉴스 검색 결과: 총 2730527건.
## - (100/1000)건 완료.
## - (200/1000)건 .
## - (300/1000)건 .
## - (400/1000)건 .
## - (500/1000)건 .
## - (600/1000)건 .
## - (700/1000)건 .
## - (800/1000)건 .
## - (900/1000)건 .
## - (1000/1000)건 .library(tm) ## Loading required package: NLP##
## Attaching package: 'NLP'## The following object is masked from 'package:httr':
##
## content#devtools::install_github('haven-jeon/KoNLP')
library(KoNLP)## Checking user defined dictionary!useSejongDic()## Backup was just finished!
## 370957 words dictionary was built.Noun extraction
sentence<- "과일 비싸다고 안 사 먹으면 죽어서 제사상에서나 먹겠지. 인생 별거 있나 나에게 남은 여름은 100번도 안 된다."
extractNoun(sentence)## [1] "과일" "제사상" "인생" "별거" "나" "남" "여름" "100"
## [9] "번"break down by morpheme
tidytext::unnest_tokens(dataframe, new column, source column, criteria)
break down text data by token (vowel / consonant / letter / morpheme /
word / n-gram /..)
#install.packages("tidytext")
library(tidytext)
hiring_date_word<- hiring_date %>%
unnest_tokens(title_word, title_text, "words")
hiring_date_wordhiring_date_pos <-hiring_date%>% unnest_tokens(pos, title_text, SimplePos09) %>%
group_by(originallink) %>%
mutate(pos_order = 1:n())
hiring_date_posLet’s take nouns only.
hiring_date_pos2<- hiring_date_pos %>%
filter(str_detect(pos, "/n")) %>%
mutate(pos_done = str_remove(pos, "/n.*$")) %>%
mutate(pos_done = str_remove(pos_done, ".*\\+(?=.)"))
hiring_date_pos2